<!DOCTYPE html>
<html lang=zh>
<head>
    <!-- so meta -->
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="HandheldFriendly" content="True">
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
    <meta name="description" content="开发环境 python: 3.7系统: windows10编辑器: PyCharm Community  文档地址 Scrapy官方文档：http:&#x2F;&#x2F;doc.scrapy.org&#x2F;en&#x2F;latest Scrapy中文文档：http:&#x2F;&#x2F;scrapy-chs.readthedocs.io&#x2F;zh_CN&#x2F;latest&#x2F;index.html  Demo地址 https:&#x2F;&#x2F;gitee.com&#x2F;fengz">
<meta property="og:type" content="article">
<meta property="og:title" content="[python爬虫]Scrapy入门使用教程">
<meta property="og:url" content="https:&#x2F;&#x2F;fengzxia.gitee.io&#x2F;posts&#x2F;ae675e47.html">
<meta property="og:site_name" content="只是学习学习的博客">
<meta property="og:description" content="开发环境 python: 3.7系统: windows10编辑器: PyCharm Community  文档地址 Scrapy官方文档：http:&#x2F;&#x2F;doc.scrapy.org&#x2F;en&#x2F;latest Scrapy中文文档：http:&#x2F;&#x2F;scrapy-chs.readthedocs.io&#x2F;zh_CN&#x2F;latest&#x2F;index.html  Demo地址 https:&#x2F;&#x2F;gitee.com&#x2F;fengz">
<meta property="og:locale" content="zh_CN">
<meta property="article:published_time" content="2019-07-11T08:52:53.000Z">
<meta property="article:modified_time" content="2020-04-16T12:11:09.194Z">
<meta property="article:author" content="只是学习学习">
<meta property="article:tag" content="Spider">
<meta name="twitter:card" content="summary">
    
    
        
          
              <link rel="shortcut icon" href="/images/favicon.ico">
          
        
        
          
            <link rel="icon" type="image/png" href="/images/favicon-192x192.png" sizes="192x192">
          
        
        
          
            <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon.png">
          
        
    
    <!-- title -->
    <title>[python爬虫]Scrapy入门使用教程</title>
    <!-- styles -->
    
<link rel="stylesheet" href="/css/style.css">

    <!-- persian styles -->
    
      
<link rel="stylesheet" href="/css/rtl.css">

    
    <!-- rss -->
    
    
    
<meta name="generator" content="Hexo 4.1.1"></head>

<body class="max-width mx-auto px3 ltr">
    
      <div id="header-post">
  <a id="menu-icon" href="#"><i class="fas fa-bars fa-lg"></i></a>
  <a id="menu-icon-tablet" href="#"><i class="fas fa-bars fa-lg"></i></a>
  <a id="top-icon-tablet" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');" style="display:none;"><i class="fas fa-chevron-up fa-lg"></i></a>
  <span id="menu">
    <span id="nav">
      <ul>
         
          <li><a href="/">主页</a></li>
         
          <li><a href="/archives/">归档</a></li>
         
          <li><a href="/categories/">分类</a></li>
         
          <li><a href="/tags/">标签</a></li>
         
          <li><a href="/search/">搜索</a></li>
         
          <li><a href="/about/">关于</a></li>
         
          <li><a href="/friend_ship/">友链</a></li>
        
      </ul>
    </span>
    <br/>
    <span id="actions">
      <ul>
        
        <li><a class="icon" href="/posts/89d31750.html"><i class="fas fa-chevron-left" aria-hidden="true" onmouseover="$('#i-prev').toggle();" onmouseout="$('#i-prev').toggle();"></i></a></li>
        
        
        <li><a class="icon" href="/posts/3efb331f.html"><i class="fas fa-chevron-right" aria-hidden="true" onmouseover="$('#i-next').toggle();" onmouseout="$('#i-next').toggle();"></i></a></li>
        
        <li><a class="icon" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');"><i class="fas fa-chevron-up" aria-hidden="true" onmouseover="$('#i-top').toggle();" onmouseout="$('#i-top').toggle();"></i></a></li>
        <li><a class="icon" href="#"><i class="fas fa-share-alt" aria-hidden="true" onmouseover="$('#i-share').toggle();" onmouseout="$('#i-share').toggle();" onclick="$('#share').toggle();return false;"></i></a></li>
      </ul>
      <span id="i-prev" class="info" style="display:none;">上一篇</span>
      <span id="i-next" class="info" style="display:none;">下一篇</span>
      <span id="i-top" class="info" style="display:none;">返回顶部</span>
      <span id="i-share" class="info" style="display:none;">分享文章</span>
    </span>
    <br/>
    <div id="share" style="display: none">
      <ul>
  <li><a class="icon" href="http://www.facebook.com/sharer.php?u=https://fengzxia.gitee.io/posts/ae675e47.html" target="_blank" rel="noopener"><i class="fab fa-facebook " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://twitter.com/share?url=https://fengzxia.gitee.io/posts/ae675e47.html&text=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-twitter " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.linkedin.com/shareArticle?url=https://fengzxia.gitee.io/posts/ae675e47.html&title=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-linkedin " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://pinterest.com/pin/create/bookmarklet/?url=https://fengzxia.gitee.io/posts/ae675e47.html&is_video=false&description=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-pinterest " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="mailto:?subject=[python爬虫]Scrapy入门使用教程&body=Check out this article: https://fengzxia.gitee.io/posts/ae675e47.html"><i class="fas fa-envelope " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://getpocket.com/save?url=https://fengzxia.gitee.io/posts/ae675e47.html&title=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-get-pocket " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://reddit.com/submit?url=https://fengzxia.gitee.io/posts/ae675e47.html&title=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-reddit " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.stumbleupon.com/submit?url=https://fengzxia.gitee.io/posts/ae675e47.html&title=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-stumbleupon " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://digg.com/submit?url=https://fengzxia.gitee.io/posts/ae675e47.html&title=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-digg " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.tumblr.com/share/link?url=https://fengzxia.gitee.io/posts/ae675e47.html&name=[python爬虫]Scrapy入门使用教程&description=" target="_blank" rel="noopener"><i class="fab fa-tumblr " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://news.ycombinator.com/submitlink?u=https://fengzxia.gitee.io/posts/ae675e47.html&t=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-hacker-news " aria-hidden="true"></i></a></li>
</ul>

    </div>
    <div id="toc">
      <ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#开发环境"><span class="toc-number">1.</span> <span class="toc-text">开发环境</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#文档地址"><span class="toc-number">2.</span> <span class="toc-text">文档地址</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Demo地址"><span class="toc-number">3.</span> <span class="toc-text">Demo地址</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#安装"><span class="toc-number">4.</span> <span class="toc-text">安装</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#快速入门"><span class="toc-number">5.</span> <span class="toc-text">快速入门</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#1-创建项目"><span class="toc-number">5.1.</span> <span class="toc-text">1. 创建项目</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#2-目录结构介绍"><span class="toc-number">5.2.</span> <span class="toc-text">2. 目录结构介绍</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3-使用Scrapy框架爬取糗事百科"><span class="toc-number">5.3.</span> <span class="toc-text">3. 使用Scrapy框架爬取糗事百科</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#4-完整的爬虫代码"><span class="toc-number">5.3.1.</span> <span class="toc-text">4. 完整的爬虫代码</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#提取数据"><span class="toc-number">5.4.</span> <span class="toc-text">提取数据</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#Selectors选择器简介"><span class="toc-number">5.4.1.</span> <span class="toc-text">Selectors选择器简介</span></a></li></ol></li></ol></li></ol>
    </div>
  </span>
</div>

    
    <div class="content index py4">
        
        <article class="post" itemscope itemtype="http://schema.org/BlogPosting">
  <header>
    
    <h1 class="posttitle" itemprop="name headline">
        [python爬虫]Scrapy入门使用教程
    </h1>



    <div class="meta">
      <span class="author" itemprop="author" itemscope itemtype="http://schema.org/Person">
        <span itemprop="name">只是学习学习的博客</span>
      </span>
      
    <div class="postdate">
      
        <time datetime="2019-07-11T08:52:53.000Z" itemprop="datePublished">2019-07-11</time>
        
      
    </div>


      
    <div class="article-category">
        <i class="fas fa-archive"></i>
        <a class="category-link" href="/categories/Python/">Python</a> › <a class="category-link" href="/categories/Python/%E7%88%AC%E8%99%AB/">爬虫</a> › <a class="category-link" href="/categories/Python/%E7%88%AC%E8%99%AB/Scrapy/">Scrapy</a>
    </div>


      
    <div class="article-tag">
        <i class="fas fa-tag"></i>
        <a class="tag-link" href="/tags/Spider/" rel="tag">Spider</a>
    </div>


    </div>
  </header>
  

  <div class="content"itemprop ="articleBody">
    <h2 id="开发环境"><a href="#开发环境" class="headerlink" title="开发环境"></a>开发环境</h2><blockquote>
<p>python: 3.7<br>系统: windows10<br>编辑器: PyCharm Community</p>
</blockquote>
<h2 id="文档地址"><a href="#文档地址" class="headerlink" title="文档地址"></a>文档地址</h2><ul>
<li>Scrapy官方文档：<a href="http://doc.scrapy.org/en/latest" target="_blank" rel="noopener">http://doc.scrapy.org/en/latest</a></li>
<li>Scrapy中文文档：<a href="http://scrapy-chs.readthedocs.io/zh_CN/latest/index.html" target="_blank" rel="noopener">http://scrapy-chs.readthedocs.io/zh_CN/latest/index.html</a></li>
</ul>
<h2 id="Demo地址"><a href="#Demo地址" class="headerlink" title="Demo地址"></a>Demo地址</h2><blockquote>
<p><a href="https://gitee.com/fengzxia/python_crawler_learning/tree/master/scrapy/qiushibaike" target="_blank" rel="noopener">https://gitee.com/fengzxia/python_crawler_learning/tree/master/scrapy/qiushibaike</a></p>
</blockquote>
<h2 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h2><p>通过 <code>pip install scrapy</code> 即可快速安装</p>
<blockquote>
<p>注意：</p>
<ol>
<li>在ubuntu上安装scrapy之前，需要先执行下边的命令安装依赖：<pre><code><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">sudo apt-get install python3-dev build-essential python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev</span><br></pre></td></tr></table></figure></code></pre> 然后再通过 <code>pip install scrapy</code> 安装</li>
<li>如果在windows系统下，提示这个错误<code>ModuleNotFoundError: No module named &#39;win32api&#39;</code>，那么使用以下命令可以解决：<code>pip install pypiwin32</code></li>
</ol>
</blockquote>
<h2 id="快速入门"><a href="#快速入门" class="headerlink" title="快速入门"></a>快速入门</h2><h3 id="1-创建项目"><a href="#1-创建项目" class="headerlink" title="1. 创建项目"></a>1. 创建项目</h3><p>要使用Scrapy框架创建项目，需要通过命令来创建。首先进入到你想把这个项目存放的目录。然后使用以下命令创建：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">scrapy startproject [项目名称]</span><br></pre></td></tr></table></figure>
<h3 id="2-目录结构介绍"><a href="#2-目录结构介绍" class="headerlink" title="2. 目录结构介绍"></a>2. 目录结构介绍</h3><p>以下介绍主要文件的作用</p>
<ol>
<li><code>items.py</code>: 用来存放爬虫爬取下来数据的模型</li>
<li><code>middlewares.py</code>: 用来存放各种中间件的文件</li>
<li><code>pipelines.py</code>: 用来将items的模型存储到本地磁盘中</li>
<li><code>settings.py</code>: 本爬虫的一些配置信息（比如请求头、多久发送一次请求、ip代理池等）</li>
<li><code>scrapy.cfg</code>: 项目的配置文件</li>
<li><code>spiders包</code>: 以后所有的爬虫，都是存放到这个里面</li>
</ol>
<h3 id="3-使用Scrapy框架爬取糗事百科"><a href="#3-使用Scrapy框架爬取糗事百科" class="headerlink" title="3. 使用Scrapy框架爬取糗事百科"></a>3. 使用Scrapy框架爬取糗事百科</h3><ol>
<li><p>使用命令创建一个爬虫</p>
<pre><code><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">scrapy startproject qsbk</span><br></pre></td></tr></table></figure>
创建了一个名叫qsbk的爬虫</code></pre></li>
<li><p>修改<code>settings.py</code>代码</p>
<pre><code>**在做爬虫之前，一定要记得修改`setting.py`中的设置，下边有两个地方是强烈建议设置的**
1. `ROBOTSTXT_OBEY`: 设置为False。默认是True。即遵守机器协议，那么在爬虫的时候，scrapy首先去找`robots.txt`文件，如果没有找到。则直接停止爬取。
2. `USER_AGENT`: 设置请求头，将爬虫伪装成正常请求，不设置直接请求会报错，错误信息如下</code></pre> <figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">2019-07-11 16:08:58 [scrapy.core.scraper] ERROR: Error downloading &lt;GET https:&#x2F;&#x2F;www.qiushibaike.com&#x2F;text&#x2F;1&gt;</span><br><span class="line">Traceback (most recent call last):</span><br><span class="line">  File &quot;C:\Users\fengz\AppData\Local\Programs\Python\Python37\lib\site-packages\scrapy\core\downloader\middleware.py&quot;, line 43, in process_request</span><br><span class="line">    defer.returnValue((yield download_func(request&#x3D;request,spider&#x3D;spider)))</span><br><span class="line">twisted.web._newclient.ResponseNeverReceived: [&lt;twisted.python.failure.Failure twisted.internet.error.ConnectionDone: Connection was closed cleanly.&gt;]</span><br></pre></td></tr></table></figure>
<h4 id="4-完整的爬虫代码"><a href="#4-完整的爬虫代码" class="headerlink" title="4. 完整的爬虫代码"></a>4. 完整的爬虫代码</h4></li>
<li><p><code>items.py</code> 部分代码</p>
 <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">QsbkItem</span><span class="params">(scrapy.Item)</span>:</span></span><br><span class="line">    author = scrapy.Field()</span><br><span class="line">    content = scrapy.Field()</span><br></pre></td></tr></table></figure></li>
<li><p><code>spiders</code>包下新建一个爬虫脚本 <code>qsbk_spider.py</code> 代码如下</p>
 <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"><span class="keyword">from</span> qsbk.items <span class="keyword">import</span> QsbkItem</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">QsbkSpider</span><span class="params">(scrapy.Spider)</span>:</span></span><br><span class="line">    name = <span class="string">"qsbk"</span></span><br><span class="line">    allowed_domains = [<span class="string">'qiushibaike.com'</span>]</span><br><span class="line">    start_urls = [<span class="string">'https://www.qiushibaike.com/text/1'</span>]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span><span class="params">(self, response)</span>:</span></span><br><span class="line">        outerbox = response.xpath(<span class="string">"//div[@id='content-left']/div"</span>)</span><br><span class="line">        items = []</span><br><span class="line">        <span class="keyword">for</span> box <span class="keyword">in</span> outerbox:</span><br><span class="line">            author = str(box.xpath(<span class="string">"div[@class='author clearfix']/a/h2/text()"</span>).extract_first()).strip()</span><br><span class="line">            content = str(box.xpath(<span class="string">"a/div[@class='content']/span/text()"</span>).extract_first()).strip()</span><br><span class="line">            item = QsbkItem()</span><br><span class="line">            item[<span class="string">'author'</span>] = author</span><br><span class="line">            item[<span class="string">'content'</span>] = content</span><br><span class="line">            items.append(item)</span><br><span class="line">        <span class="keyword">return</span> items</span><br></pre></td></tr></table></figure>
<p> 这里也可以通过输入 <code>scrapy genspider [爬虫名字] [域名]</code> 在 <code>spiders</code>包下来生成爬虫相关代码：</p>
<ol>
<li>通过命令生成爬虫代码 <figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">scrapy genspider qsbk www.qiushibaike.com</span><br></pre></td></tr></table></figure></li>
<li>生成的代码如下，然后再稍作修改 <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Qsbk</span> <span class="title">Spider</span><span class="params">(scrapy.Spider)</span>:</span></span><br><span class="line">    name = <span class="string">'qsbk '</span></span><br><span class="line">    allowed_domains = [<span class="string">'www.qiushibaike.com'</span>]</span><br><span class="line">    start_urls = [<span class="string">'http://www.qiushibaike.com/'</span>]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span><span class="params">(self, response)</span>:</span></span><br><span class="line">        <span class="keyword">pass</span></span><br></pre></td></tr></table></figure></li>
</ol>
<ul>
<li><p><code>name</code>: 这个爬虫的名字，名字必须是唯一的</p>
</li>
<li><p><code>allow_domains</code>: 允许的域名。爬虫只会爬取这个域名下的网页，其他不是这个域名下的网页会被自动忽略</p>
</li>
<li><p><code>start_urls</code>: 爬虫从这个变量中的url开始</p>
</li>
<li><p><code>parse方法</code>: 引擎会把下载器下载回来的数据扔给爬虫解析，爬虫再把数据传给这个parse方法。这个是个固定的写法。这个方法的作用有两个，第一个是提取想要的数据。第二个是生成下一个请求的url</p>
</li>
<li><p>这里使用的是<code>xpath</code>方式提取的数据</p>
</li>
<li><p><code>extract_first()</code>: 获取通过xpath提取的第一条数据，还有 <code>extract()</code>、<code>getall()</code>、<code>get()</code>写法</p>
<table>
<thead>
<tr>
<th align="center">方法名</th>
<th align="left">描述</th>
</tr>
</thead>
<tbody><tr>
<td align="center"><code>extract()</code></td>
<td align="left">返回的是一个list，里面包含了多个string，<br>如果只有一个string，则返回[‘我很孤独’]这样的形式</td>
</tr>
<tr>
<td align="center"><code>extract_first()</code></td>
<td align="left">返回的是string，list里面第一个string</td>
</tr>
<tr>
<td align="center"><code>getall()</code></td>
<td align="left">同<code>extract()</code>一样返回的是list</td>
</tr>
<tr>
<td align="center"><code>get()</code></td>
<td align="left">同<code>extract_first()</code>一样返回的是string</td>
</tr>
</tbody></table>
</li>
<li><p><code>strip()</code>: 去除字符串中的空格</p>
</li>
<li><p><code>str()</code>: 将内容转为 <code>字符串</code> 类型，因为有些时候 <code>extract_first()</code> 返回的类型不匹配，不能使用<code>strip()</code></p>
<blockquote>
<p>Scrapy为<code>QsbkSpider</code>的 <code>start_urls</code> 属性中的每个URL创建了 <code>scrapy.Request</code> 对象，并将 <code>parse</code> 方法作为回调函数(<code>callback</code>)赋值给了<code>Request</code>。<br><code>Request对象</code>经过调度，执行生成 <code>scrapy.http.Response</code> 对象并送回给<code>spider parse()</code> 方法。</p>
</blockquote>
<h3 id="提取数据"><a href="#提取数据" class="headerlink" title="提取数据"></a>提取数据</h3><h4 id="Selectors选择器简介"><a href="#Selectors选择器简介" class="headerlink" title="Selectors选择器简介"></a>Selectors选择器简介</h4><p>从网页中提取数据有很多方法。Scrapy使用了一种基于 <a href="http://www.w3.org/TR/xpath" target="_blank" rel="noopener">XPath</a> 和 <a href="http://www.w3.org/TR/selectors" target="_blank" rel="noopener">CSS</a> 表达式机制: <a href="https://scrapy-chs.readthedocs.io/zh_CN/latest/topics/selectors.html#topics-selectors" target="_blank" rel="noopener">Scrapy Selectors</a>。 关于selector和其他提取机制的信息请参考 <a href="https://scrapy-chs.readthedocs.io/zh_CN/latest/topics/selectors.html#topics-selectors" target="_blank" rel="noopener">Selector文档</a></p>
<p>这里给出XPath表达式的例子及对应的含义:</p>
</li>
<li><p><code>/html/head/title</code>: 选择HTML文档中 <code>&lt;head&gt;</code> 标签内的 <code>&lt;title&gt;</code> 元素</p>
</li>
<li><p><code>/html/head/title/text()</code>: 选择上面提到的 <code>&lt;title&gt;</code> 元素的文字</p>
</li>
<li><p><code>//td</code>: 选择所有的 <code>&lt;td&gt;</code> 元素</p>
</li>
<li><p><code>//div[@class=&quot;mine&quot;]</code>: 选择所有具有 <code>class=&quot;mine&quot;</code> 属性的 div 元素</p>
<p>上边仅仅是几个简单的XPath例子，XPath实际上要比这远远强大的多。 如果您想了解的更多，我们推荐 <a href="http://www.w3school.com.cn/xpath/index.asp" target="_blank" rel="noopener">这篇XPath教程</a> 。</p>
<p>为了配合XPath，Scrapy除了提供了 <code>Selector</code> 之外，还提供了方法来避免每次从response中提取数据时生成selector的麻烦。</p>
<p>Selector有四个基本的方法(点击相应的方法可以看到详细的API文档):</p>
</li>
<li><p><code>xpath()</code>: 传入xpath表达式，返回该表达式所对应的所有节点的selector list列表 。</p>
</li>
<li><p><code>css()</code>: 传入CSS表达式，返回该表达式所对应的所有节点的selector list列表.</p>
</li>
<li><p><code>extract()</code>: 序列化该节点为unicode字符串并返回list。</p>
</li>
<li><p><code>re()</code>: 根据传入的正则表达式对数据进行提取，返回unicode字符串list列表。</p>
<p>以上 <code>Selectors选择器</code> 内容摘抄自<a href="https://scrapy-chs.readthedocs.io/zh_CN/latest/intro/tutorial.html#selectors" target="_blank" rel="noopener">Scrapy中文文档</a></p>
</li>
</ul>
</li>
<li><p><code>pipelines.py</code>部分代码</p>
 <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> json</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">QsbkPipeline</span><span class="params">(object)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line">        self.items = []</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span><span class="params">(self, item, spider)</span>:</span></span><br><span class="line">        self.items.append(dict(item))</span><br><span class="line">        print(<span class="string">"="</span> * <span class="number">40</span>)</span><br><span class="line">        <span class="keyword">return</span> item</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">close_spider</span><span class="params">(self, spider)</span>:</span></span><br><span class="line">        <span class="keyword">with</span> open(<span class="string">'qsbk.json'</span>, <span class="string">'w'</span>, encoding=<span class="string">'utf-8'</span>) <span class="keyword">as</span> fp:</span><br><span class="line">            json.dump(self.items, fp, ensure_ascii=<span class="literal">False</span>)</span><br></pre></td></tr></table></figure>
<ul>
<li><p><code>process_item</code>: 每个item pipeline组件都需要调用该方法，这个方法必须返回一个 <code>Item</code> (或任何继承类)对象， 或是抛出 <code>DropItem</code> 异常，被丢弃的item将不会被之后的pipeline组件所处理。这里是将<code>item</code>数据存入<code>items</code>集合中</p>
</li>
<li><p><code>close_spider</code>: 当spider被关闭时，这个方法被调用。这里是将<code>items</code>集合中的数据转为json字符串保存到 <code>qsbk.json</code> 文件中</p>
</li>
<li><p><em>要想<code>pipelines.py</code>中的代码执行，必须在<code>settings.py</code>中设置<code>ITEM_PIPELINES属性</code>, 否则不会执行*</em></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">ITEM_PIPELINES = &#123;</span><br><span class="line">    <span class="string">'qsbk.pipelines.QsbkPipeline'</span>: <span class="number">300</span>,</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure></li>
</ul>
</li>
<li><p>运行scrapy项目<br> 运行scrapy项目。需要在终端，进入项目所在的路径，然后 <code>scrapy crawl [爬虫名字]</code> 即可运行指定的爬虫。如果不想每次都在命令行中运行，那么可以把这个命令写在一个文件中。以后就在pycharm中执行运行这个文件就可以了。比如现在新创建一个文件叫做 <code>start.py</code>，然后在这个文件中填入以下代码</p>
 <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> cmdline</span><br><span class="line"></span><br><span class="line">cmdline.execute(<span class="string">'scrapy crawl qsbk'</span>.split())</span><br></pre></td></tr></table></figure></li>
</ol>

<div id="gitalk-container"></div>
<script src="https://cdn.bootcss.com/blueimp-md5/2.12.0/js/md5.min.js"></script><link rel="stylesheet" href="https://unpkg.com/gitalk/dist/gitalk.css"><script src="https://unpkg.com/gitalk/dist/gitalk.min.js"></script>

		<script>
		var gitalkConfig = {"clientID":"28a065e3b5afee156f4f","clientSecret":"e18104d6f3219526491fc42c892ff6a26b8f3e74","repo":"blog_comment","owner":"FENG1000","admin":["FENG1000"],"distractionFreeMode":false};
	    gitalkConfig.id = md5(location.pathname);
		var gitalk = new Gitalk(gitalkConfig);
	    gitalk.render("gitalk-container");
	    </script>
    
  <blockquote>
      <p>
          作者: 只是学习学习<br>
          邮箱: fengzxia1000@163.com<br>
          原文地址: <a href="https://fengzxia.gitee.io/posts/ae675e47.html" target="_blank" rel="noopener">https://fengzxia.gitee.io/posts/ae675e47.html</a><br>
          版权声明: 商业转载请联系作者获得授权，非商业转载请注明出处。
      </p>
  </blockquote>

  <script>
      function setClipboardText(event){
          // clipboardData 对象是为通过编辑菜单、快捷菜单和快捷键执行的编辑操作所保留的，也就是你复制或者剪切内容
          let clipboardData = event.clipboardData || window.clipboardData;
          // 如果未复制或者未剪切，则return出去
          if (!clipboardData) { return; }
          event.preventDefault();
          // Selection 对象，表示用户选择的文本范围或光标的当前位置。
          //     声明一个变量接收 -- 用户输入的剪切或者复制的文本转化为字符串
          let text = window.getSelection().toString();

          if (text) {
            // 如果文本存在则先取消文本默认事件
            event.preventDefault();
            // 通过调用常clipboardData对象的 setData(format, data) 方法；来设置相关文本
            // format: 一个DOMString 表示要添加到 drag object的拖动数据的类型。
            // data: 一个 DOMString表示要添加到 drag object的数据。
            var copyright = '\n\n'
                            + '\n著作权归作者所有。'
                            + '\n商业转载请联系作者获得授权，非商业转载请注明出处。'
                            + '\n作者: 只是学习学习'
                            + '\n邮箱: fengzxia1000@163.com'
                            + '\n原文地址: https://fengzxia.gitee.io/posts/ae675e47.html'

            clipboardData.setData('text/plain', text + copyright);

          }
      };
      var contents = document.getElementsByClassName("content");
      // 监听文章内容的copy事件
      contents[0].addEventListener('copy',function(e){
          setClipboardText(e);
      });
  </script>

  </div>
</article>




        
          <div id="footer-post-container">
  <div id="footer-post">

    <div id="nav-footer" style="display: none">
      <ul>
         
          <li><a href="/">主页</a></li>
         
          <li><a href="/archives/">归档</a></li>
         
          <li><a href="/categories/">分类</a></li>
         
          <li><a href="/tags/">标签</a></li>
         
          <li><a href="/search/">搜索</a></li>
         
          <li><a href="/about/">关于</a></li>
         
          <li><a href="/friend_ship/">友链</a></li>
        
      </ul>
    </div>

    <div id="toc-footer" style="display: none">
      <ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#开发环境"><span class="toc-number">1.</span> <span class="toc-text">开发环境</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#文档地址"><span class="toc-number">2.</span> <span class="toc-text">文档地址</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Demo地址"><span class="toc-number">3.</span> <span class="toc-text">Demo地址</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#安装"><span class="toc-number">4.</span> <span class="toc-text">安装</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#快速入门"><span class="toc-number">5.</span> <span class="toc-text">快速入门</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#1-创建项目"><span class="toc-number">5.1.</span> <span class="toc-text">1. 创建项目</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#2-目录结构介绍"><span class="toc-number">5.2.</span> <span class="toc-text">2. 目录结构介绍</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3-使用Scrapy框架爬取糗事百科"><span class="toc-number">5.3.</span> <span class="toc-text">3. 使用Scrapy框架爬取糗事百科</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#4-完整的爬虫代码"><span class="toc-number">5.3.1.</span> <span class="toc-text">4. 完整的爬虫代码</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#提取数据"><span class="toc-number">5.4.</span> <span class="toc-text">提取数据</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#Selectors选择器简介"><span class="toc-number">5.4.1.</span> <span class="toc-text">Selectors选择器简介</span></a></li></ol></li></ol></li></ol>
    </div>

    <div id="share-footer" style="display: none">
      <ul>
  <li><a class="icon" href="http://www.facebook.com/sharer.php?u=https://fengzxia.gitee.io/posts/ae675e47.html" target="_blank" rel="noopener"><i class="fab fa-facebook fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://twitter.com/share?url=https://fengzxia.gitee.io/posts/ae675e47.html&text=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-twitter fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.linkedin.com/shareArticle?url=https://fengzxia.gitee.io/posts/ae675e47.html&title=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-linkedin fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://pinterest.com/pin/create/bookmarklet/?url=https://fengzxia.gitee.io/posts/ae675e47.html&is_video=false&description=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-pinterest fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="mailto:?subject=[python爬虫]Scrapy入门使用教程&body=Check out this article: https://fengzxia.gitee.io/posts/ae675e47.html"><i class="fas fa-envelope fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://getpocket.com/save?url=https://fengzxia.gitee.io/posts/ae675e47.html&title=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-get-pocket fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://reddit.com/submit?url=https://fengzxia.gitee.io/posts/ae675e47.html&title=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-reddit fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.stumbleupon.com/submit?url=https://fengzxia.gitee.io/posts/ae675e47.html&title=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-stumbleupon fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://digg.com/submit?url=https://fengzxia.gitee.io/posts/ae675e47.html&title=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-digg fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.tumblr.com/share/link?url=https://fengzxia.gitee.io/posts/ae675e47.html&name=[python爬虫]Scrapy入门使用教程&description=" target="_blank" rel="noopener"><i class="fab fa-tumblr fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://news.ycombinator.com/submitlink?u=https://fengzxia.gitee.io/posts/ae675e47.html&t=[python爬虫]Scrapy入门使用教程" target="_blank" rel="noopener"><i class="fab fa-hacker-news fa-lg" aria-hidden="true"></i></a></li>
</ul>

    </div>

    <div id="actions-footer">
        <a id="menu" class="icon" href="#" onclick="$('#nav-footer').toggle();return false;"><i class="fas fa-bars fa-lg" aria-hidden="true"></i> 菜单</a>
        <a id="toc" class="icon" href="#" onclick="$('#toc-footer').toggle();return false;"><i class="fas fa-list fa-lg" aria-hidden="true"></i> 目录</a>
        <a id="share" class="icon" href="#" onclick="$('#share-footer').toggle();return false;"><i class="fas fa-share-alt fa-lg" aria-hidden="true"></i> 分享</a>
        <a id="top" style="display:none" class="icon" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');"><i class="fas fa-chevron-up fa-lg" aria-hidden="true"></i> 返回顶部</a>
    </div>

  </div>
</div>

        
        <footer id="footer">
  <div class="footer-left">
    Copyright &copy;
    
    
    2019-2020
    只是学习学习
  </div>
  <div class="footer-right">
    <nav>
      <ul>
         
          <li><a href="/">主页</a></li>
         
          <li><a href="/archives/">归档</a></li>
         
          <li><a href="/categories/">分类</a></li>
         
          <li><a href="/tags/">标签</a></li>
         
          <li><a href="/search/">搜索</a></li>
         
          <li><a href="/about/">关于</a></li>
         
          <li><a href="/friend_ship/">友链</a></li>
        
      </ul>
    </nav>
  </div>
</footer>

    </div>
    <!-- styles -->

<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">


<link rel="stylesheet" href="/lib/justified-gallery/css/justifiedGallery.min.css">


    <!-- jquery -->

<script src="/lib/jquery/jquery.min.js"></script>


<script src="/lib/justified-gallery/js/jquery.justifiedGallery.min.js"></script>

<!-- clipboard -->

  
<script src="/lib/clipboard/clipboard.min.js"></script>

  <script type="text/javascript">
  $(function() {
    // copy-btn HTML
    var btn = "<span class=\"btn-copy tooltipped tooltipped-sw\" aria-label=\"复制到粘贴板!\">";
    btn += '<i class="far fa-clone"></i>';
    btn += '</span>'; 
    // mount it!
    $(".highlight table").before(btn);
    var clip = new ClipboardJS('.btn-copy', {
      text: function(trigger) {
        return Array.from(trigger.nextElementSibling.querySelectorAll('.code')).reduce((str,it)=>str+it.innerText+'\n','')
      }
    });
    clip.on('success', function(e) {
      e.trigger.setAttribute('aria-label', "复制成功!");
      e.clearSelection();
    })
  })
  </script>


<script src="/js/main.js"></script>

<!-- search -->

<!-- Google Analytics -->

<!-- Baidu Analytics -->

    <script type="text/javascript">
        var _hmt = _hmt || [];
        (function() {
            var hm = document.createElement("script");
            hm.src = "https://hm.baidu.com/hm.js?54382acbd8c4bc66535bce135e0ac166";
            var s = document.getElementsByTagName("script")[0];
            s.parentNode.insertBefore(hm, s);
        })();
    </script>

<!-- Disqus Comments -->


</body>
<script>
(function(){
    var bp = document.createElement('script');
    var curProtocol = window.location.protocol.split(':')[0];
    if (curProtocol === 'https'){
   bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
  }
  else{
  bp.src = 'http://push.zhanzhang.baidu.com/push.js';
  }
    var s = document.getElementsByTagName("script")[0];
    s.parentNode.insertBefore(bp, s);
})();
</script>
</html>
